diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -9,7 +9,8 @@ python3-setuptools, python3-swh.core, python3-swh.storage (>= 0.0.33~), - python3-vcversioner + python3-vcversioner, + libjs-cryptojs Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DWUI/ diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,6 @@ # Test dependencies #Flask-Testing #blinker + +# Non-Python dependencies +# libjs-cryptojs diff --git a/resources/test/webapp.ini b/resources/test/webapp.ini --- a/resources/test/webapp.ini +++ b/resources/test/webapp.ini @@ -1,6 +1,6 @@ [main] # the dedicated storage arguments (comma separated list of values) -storage_args = http://localhost:5000/ +storage_args = http://uffizi:5002/ # either remote_storage or local_storage storage_class = remote_storage diff --git a/swh/web/ui/backend.py b/swh/web/ui/backend.py --- a/swh/web/ui/backend.py +++ b/swh/web/ui/backend.py @@ -54,6 +54,17 @@ return main.storage().content_find_occurrence({algo: hash_bin}) +def content_missing_per_sha1(sha1s): + """List content missing from storage based on sha1 + + Args: + sha1s: Iterable of sha1 to check for absence + Returns: + an iterable of sha1s missing from the storage + """ + return main.storage().content_missing_per_sha1(sha1s) + + def directory_get(sha1_bin): """Retrieve information on one directory. diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py --- a/swh/web/ui/service.py +++ b/swh/web/ui/service.py @@ -10,6 +10,26 @@ from swh.web.ui.exc import NotFoundExc +def lookup_multiple_hashes(hashes): + """Lookup the passed hashes in a single DB connection, using batch processing. + + Args: + An array of {filename: X, sha1: Y}, string X, hex sha1 string Y. + Returns: + The same array with elements updated with elem['found'] = true if + the hash is present in storage, elem['found'] = false if not. + """ + hashlist = [hashutil.hex_to_hash(elem['sha1']) for elem in hashes] + content_missing = backend.content_missing_per_sha1(hashlist) + missing = [hashutil.hash_to_hex(x) for x in content_missing] + for x in hashes: + x.update({'found': True}) + for h in hashes: + if h['sha1'] in missing: + h['found'] = False + return hashes + + def hash_and_search(filepath): """Hash the filepath's content as sha1, then search in storage if it exists. diff --git a/swh/web/ui/static/css/bootstrap-responsive.min.css b/swh/web/ui/static/css/bootstrap-responsive.min.css new file mode 100644 --- /dev/null +++ b/swh/web/ui/static/css/bootstrap-responsive.min.css @@ -0,0 +1,9 @@ +/*! + * Bootstrap Responsive v2.3.2 + * + * Copyright 2013 Twitter, Inc + * Licensed under the Apache License v2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Designed and built with all the love in the world by @mdo and @fat. + */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}@-ms-viewport{width:device-width}.hidden{display:none;visibility:hidden}.visible-phone{display:none!important}.visible-tablet{display:none!important}.hidden-desktop{display:none!important}.visible-desktop{display:inherit!important}@media(min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-tablet{display:inherit!important}.hidden-tablet{display:none!important}}@media(max-width:767px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-phone{display:inherit!important}.hidden-phone{display:none!important}}.visible-print{display:none!important}@media print{.visible-print{display:inherit!important}.hidden-print{display:none!important}}@media(min-width:1200px){.row{margin-left:-30px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:30px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px}.span12{width:1170px}.span11{width:1070px}.span10{width:970px}.span9{width:870px}.span8{width:770px}.span7{width:670px}.span6{width:570px}.span5{width:470px}.span4{width:370px}.span3{width:270px}.span2{width:170px}.span1{width:70px}.offset12{margin-left:1230px}.offset11{margin-left:1130px}.offset10{margin-left:1030px}.offset9{margin-left:930px}.offset8{margin-left:830px}.offset7{margin-left:730px}.offset6{margin-left:630px}.offset5{margin-left:530px}.offset4{margin-left:430px}.offset3{margin-left:330px}.offset2{margin-left:230px}.offset1{margin-left:130px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.564102564102564%}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%}.row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%}.row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%}.row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%}.row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%}.row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%}.row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%}.row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%}.row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%}.row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%}.row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%}.row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%}.row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%}.row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%}.row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%}.row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%}.row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%}.row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%}.row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%}.row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%}.row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%}.row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%}.row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%}.row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%}.row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%}.row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%}.row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%}.row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%}.row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%}.row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%}.row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%}.row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%}.row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%}.row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%}.row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:30px}input.span12,textarea.span12,.uneditable-input.span12{width:1156px}input.span11,textarea.span11,.uneditable-input.span11{width:1056px}input.span10,textarea.span10,.uneditable-input.span10{width:956px}input.span9,textarea.span9,.uneditable-input.span9{width:856px}input.span8,textarea.span8,.uneditable-input.span8{width:756px}input.span7,textarea.span7,.uneditable-input.span7{width:656px}input.span6,textarea.span6,.uneditable-input.span6{width:556px}input.span5,textarea.span5,.uneditable-input.span5{width:456px}input.span4,textarea.span4,.uneditable-input.span4{width:356px}input.span3,textarea.span3,.uneditable-input.span3{width:256px}input.span2,textarea.span2,.uneditable-input.span2{width:156px}input.span1,textarea.span1,.uneditable-input.span1{width:56px}.thumbnails{margin-left:-30px}.thumbnails>li{margin-left:30px}.row-fluid .thumbnails{margin-left:0}}@media(min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px}.span12{width:724px}.span11{width:662px}.span10{width:600px}.span9{width:538px}.span8{width:476px}.span7{width:414px}.span6{width:352px}.span5{width:290px}.span4{width:228px}.span3{width:166px}.span2{width:104px}.span1{width:42px}.offset12{margin-left:764px}.offset11{margin-left:702px}.offset10{margin-left:640px}.offset9{margin-left:578px}.offset8{margin-left:516px}.offset7{margin-left:454px}.offset6{margin-left:392px}.offset5{margin-left:330px}.offset4{margin-left:268px}.offset3{margin-left:206px}.offset2{margin-left:144px}.offset1{margin-left:82px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.7624309392265194%}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%}.row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%}.row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%}.row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%}.row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%}.row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%}.row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%}.row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%}.row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%}.row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%}.row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%}.row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%}.row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%}.row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%}.row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%}.row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%}.row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%}.row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%}.row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%}.row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%}.row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%}.row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%}.row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%}.row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%}.row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%}.row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%}.row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%}.row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%}.row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%}.row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%}.row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%}.row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%}.row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%}.row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%}.row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:710px}input.span11,textarea.span11,.uneditable-input.span11{width:648px}input.span10,textarea.span10,.uneditable-input.span10{width:586px}input.span9,textarea.span9,.uneditable-input.span9{width:524px}input.span8,textarea.span8,.uneditable-input.span8{width:462px}input.span7,textarea.span7,.uneditable-input.span7{width:400px}input.span6,textarea.span6,.uneditable-input.span6{width:338px}input.span5,textarea.span5,.uneditable-input.span5{width:276px}input.span4,textarea.span4,.uneditable-input.span4{width:214px}input.span3,textarea.span3,.uneditable-input.span3{width:152px}input.span2,textarea.span2,.uneditable-input.span2{width:90px}input.span1,textarea.span1,.uneditable-input.span1{width:28px}}@media(max-width:767px){body{padding-right:20px;padding-left:20px}.navbar-fixed-top,.navbar-fixed-bottom,.navbar-static-top{margin-right:-20px;margin-left:-20px}.container-fluid{padding:0}.dl-horizontal dt{float:none;width:auto;clear:none;text-align:left}.dl-horizontal dd{margin-left:0}.container{width:auto}.row-fluid{width:100%}.row,.thumbnails{margin-left:0}.thumbnails>li{float:none;margin-left:0}[class*="span"],.uneditable-input[class*="span"],.row-fluid [class*="span"]{display:block;float:none;width:100%;margin-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="offset"]:first-child{margin-left:0}.input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto}.controls-row [class*="span"]+[class*="span"]{margin-left:0}.modal{position:fixed;top:20px;right:20px;left:20px;width:auto;margin:0}.modal.fade{top:-100px}.modal.fade.in{top:20px}}@media(max-width:480px){.nav-collapse{-webkit-transform:translate3d(0,0,0)}.page-header h1 small{display:block;line-height:20px}input[type="checkbox"],input[type="radio"]{border:1px solid #ccc}.form-horizontal .control-label{float:none;width:auto;padding-top:0;text-align:left}.form-horizontal .controls{margin-left:0}.form-horizontal .control-list{padding-top:0}.form-horizontal .form-actions{padding-right:10px;padding-left:10px}.media .pull-left,.media .pull-right{display:block;float:none;margin-bottom:10px}.media-object{margin-right:0;margin-left:0}.modal{top:10px;right:10px;left:10px}.modal-header .close{padding:10px;margin:-10px}.carousel-caption{position:static}}@media(max-width:979px){body{padding-top:0}.navbar-fixed-top,.navbar-fixed-bottom{position:static}.navbar-fixed-top{margin-bottom:20px}.navbar-fixed-bottom{margin-top:20px}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px}.navbar .container{width:auto;padding:0}.navbar .brand{padding-right:10px;padding-left:10px;margin:0 0 0 -5px}.nav-collapse{clear:both}.nav-collapse .nav{float:none;margin:0 0 10px}.nav-collapse .nav>li{float:none}.nav-collapse .nav>li>a{margin-bottom:2px}.nav-collapse .nav>.divider-vertical{display:none}.nav-collapse .nav .nav-header{color:#777;text-shadow:none}.nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#777;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-collapse .dropdown-menu li+li a{margin-bottom:2px}.nav-collapse .nav>li>a:hover,.nav-collapse .nav>li>a:focus,.nav-collapse .dropdown-menu a:hover,.nav-collapse .dropdown-menu a:focus{background-color:#f2f2f2}.navbar-inverse .nav-collapse .nav>li>a,.navbar-inverse .nav-collapse .dropdown-menu a{color:#999}.navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .nav>li>a:focus,.navbar-inverse .nav-collapse .dropdown-menu a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:focus{background-color:#111}.nav-collapse.in .btn-group{padding:0;margin-top:5px}.nav-collapse .dropdown-menu{position:static;top:auto;left:auto;display:none;float:none;max-width:none;padding:0;margin:0 15px;background-color:transparent;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.nav-collapse .open>.dropdown-menu{display:block}.nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none}.nav-collapse .dropdown-menu .divider{display:none}.nav-collapse .nav>li>.dropdown-menu:before,.nav-collapse .nav>li>.dropdown-menu:after{display:none}.nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1)}.navbar-inverse .nav-collapse .navbar-form,.navbar-inverse .nav-collapse .navbar-search{border-top-color:#111;border-bottom-color:#111}.navbar .nav-collapse .nav.pull-right{float:none;margin-left:0}.nav-collapse,.nav-collapse.collapse{height:0;overflow:hidden}.navbar .btn-navbar{display:block}.navbar-static .navbar-inner{padding-right:10px;padding-left:10px}}@media(min-width:980px){.nav-collapse.collapse{height:auto!important;overflow:visible!important}} diff --git a/swh/web/ui/static/style.css b/swh/web/ui/static/css/style.css rename from swh/web/ui/static/style.css rename to swh/web/ui/static/css/style.css --- a/swh/web/ui/static/style.css +++ b/swh/web/ui/static/css/style.css @@ -25,4 +25,8 @@ .flash { background: #cee5F5; padding: 0.5em; border: 1px solid #aacbe2; } .error { background: #f0d6d6; padding: 0.5em; } +/*.dropbox { text-align: center; width: 80%; height: 10%; }*/ +.file-found { color: #23BA49; } +.file-nfound { color: #FF4747; } +} \ No newline at end of file diff --git a/swh/web/ui/static/js/filedrop.js b/swh/web/ui/static/js/filedrop.js new file mode 100644 --- /dev/null +++ b/swh/web/ui/static/js/filedrop.js @@ -0,0 +1,146 @@ +/** + * Search page management + */ + +$.fn.extend({ + /** + * Call on any HTMLElement to make that element the recipient of files + * drag & dropped into it. + * Files then have their sha1 checksum calculated + * and searched in SWH. + * Args: + * resultDiv: the table where the result should be displayed + * errorDiv: the element where the error message should be displayed + */ + filedrop: function(fileLister, searchForm) { + + return this.each(function() { + + var dragwin = $(this); + var fileshovering = false; + + dragwin.on('dragover', function(event) { + event.stopPropagation(); + event.preventDefault(); + }); + + dragwin.on('dragenter', function(event) { + event.stopPropagation(); + event.preventDefault(); + if (!fileshovering) { + dragwin.css("border-style", "solid"); + dragwin.css("box-shadow", "inset 0 3px 4px"); + fileshovering = true; + } + }); + + dragwin.on('dragover', function(event) { + event.stopPropagation(); + event.preventDefault(); + if (!fileshovering) { + dragwin.css("border-style", "solid"); + dragwin.css("box-shadow", "inset 0 3px 4px"); + fileshovering = true; + } + }); + + dragwin.on('dragleave', function(event) { + event.stopPropagation(); + event.preventDefault(); + if (fileshovering) { + dragwin.css("border-style", "dashed"); + dragwin.css("box-shadow", "none"); + fileshovering = false; + } + }); + + dragwin.on('drop', function(event) { + event.stopPropagation(); + event.preventDefault(); + if (fileshovering) { + dragwin.css("border-style", "dashed"); + dragwin.css("box-shadow", "none"); + fileshovering = false; + } + var myfiles = event.originalEvent.dataTransfer.files; + if (myfiles.length >= 1) { + handleFiles(myfiles, fileLister, searchForm); + } + }); + }); + }, + /** + * Call on a jQuery-selected input to make it sensitive to + * the reception of new files, and have it process received + * files. + * Args: + * fileLister: the element keeping track of the files + * searchForm: the form whose submission will POST the file + * information + */ + filedialog: function(fileLister, searchForm) { + return this.each(function() { + var elem = $(this); + elem.on('change', function(){ + handleFiles(this.files, fileLister, searchForm); + }); + }); + }, + /** + * Call on a jQuery-selected element to delegate its click + * event to the given input instead. + * Args: + * input: the element to be clicked when the caller is clicked. + */ + inputclick: function(input) { + return this.each(function() { + $(this).click(function(event) { + event.preventDefault(); + input.click(); + }); + }); + } +}); + + +var nameList = []; /** Avoid adding the same file twice **/ + +function handleFiles(myfiles, fileLister, searchForm) { + for (var i = 0; i < myfiles.length; i++) { + var file = myfiles.item(i); + if (nameList.indexOf(file.name) == -1) { + nameList.push(file.name); + var fr = new FileReader(); + fileLister.append(make_row(file.name)); + bind_reader(fr, file.name, searchForm); + fr.readAsArrayBuffer(file); + } + } +}; + +function bind_reader(filereader, filename, searchForm) { + filereader.onloadend = function(evt) { + if (evt.target.readyState == FileReader.DONE){ + return fileReadDone(evt.target.result, filename, searchForm); + } + }; +} + +function make_row(name) { + return "
"+name+"
"; +} + +function fileReadDone(buffer, fname, searchForm) { + + function bytesToWords(bytes) { + for (var words = [], i = 0, b = 0; i < bytes.length; i++, b += 8) + words[b >>> 5] |= bytes[i] << (24 - b % 32); + return words; + } + var wordArray = CryptoJS.lib.WordArray.create(buffer); + var sha1 = CryptoJS.SHA1(wordArray); + searchForm.append($("", {type: "hidden", + name: fname, + value: sha1} + )); +} diff --git a/swh/web/ui/static/lib/core.js b/swh/web/ui/static/lib/core.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/core.js @@ -0,0 +1 @@ +/usr/share/javascript/cryptojs/core.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/lib-typedarrays.js b/swh/web/ui/static/lib/lib-typedarrays.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/lib-typedarrays.js @@ -0,0 +1 @@ +/usr/share/javascript/cryptojs/lib-typedarrays.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/sha1.js b/swh/web/ui/static/lib/sha1.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/sha1.js @@ -0,0 +1 @@ +/usr/share/javascript/cryptojs/sha1.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/sha256.js b/swh/web/ui/static/lib/sha256.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/sha256.js @@ -0,0 +1 @@ +/usr/share/javascript/cryptojs/sha256.js \ No newline at end of file diff --git a/swh/web/ui/templates/layout.html b/swh/web/ui/templates/layout.html --- a/swh/web/ui/templates/layout.html +++ b/swh/web/ui/templates/layout.html @@ -3,17 +3,16 @@ - + {% block title %}{% endblock %} - The Software Heritage Archive + - + diff --git a/swh/web/ui/templates/upload_and_search.html b/swh/web/ui/templates/upload_and_search.html --- a/swh/web/ui/templates/upload_and_search.html +++ b/swh/web/ui/templates/upload_and_search.html @@ -1,30 +1,99 @@ {% extends "layout.html" %} -{% block title %}Search or Upload/Hash/Search{% endblock %} +{% block title %}Search SWH{% endblock %} {% block content %} -
- Input some hash: -
+ + + + + +
+
- - or some file to hash: - + +
+
+ +
+
+
+ +
+ Drag and drop or click here to hash files and search for them. + Your files will NOT be uploaded, hashing is done locally. + Filesizes over 20Mb are slow to process, we recommend obtaining their hash via + GNU sha1sum or some other online tool. +
+
+
+
+
+
- - {% if messages is not none %} + + {% if search_stats is not none and search_stats %} + + {% endif %} + {% if responses is not none and responses %} + + + + + + + {% for resp in responses %} + + + {% if resp['found'] %} + + + {% else %} + + + {% endif %} + + {% endfor %} +
File nameSHA1 hashResult
{{ resp['filename'] }}{{ resp['sha1'] }}{{ resp['sha1'] }}
+ {% endif %} + {% if messages is not none and messages %} +
{% for message in messages %} -
-
{{ message | safe }}
- {% endfor %} +
+
{{ message | safe }}
+ {% endfor %} +
{% endif %} - + +
+
{% endblock %} diff --git a/swh/web/ui/tests/test_backend.py b/swh/web/ui/tests/test_backend.py --- a/swh/web/ui/tests/test_backend.py +++ b/swh/web/ui/tests/test_backend.py @@ -135,6 +135,45 @@ {'sha1': sha1_bin}) @istest + def content_missing_per_sha1_none(self): + # given + sha1s_bin = [hashutil.hex_to_hash( + '456caf10e9535160d90e874b45aa426de762f19f'), + hashutil.hex_to_hash( + '745bab676c8f3cec8016e0c39ea61cf57e518865' + )] + self.storage.content_missing_per_sha1 = MagicMock(return_value=[]) + + # when + actual_content = backend.content_missing_per_sha1(sha1s_bin) + + # then + self.assertEquals(actual_content, []) + self.storage.content_missing_per_sha1.assert_called_with(sha1s_bin) + + @istest + def content_missing_per_sha1_some(self): + # given + sha1s_bin = [hashutil.hex_to_hash( + '456caf10e9535160d90e874b45aa426de762f19f'), + hashutil.hex_to_hash( + '745bab676c8f3cec8016e0c39ea61cf57e518865' + )] + self.storage.content_missing_per_sha1 = MagicMock(return_value=[ + hashutil.hex_to_hash( + '745bab676c8f3cec8016e0c39ea61cf57e518865' + )]) + + # when + actual_content = backend.content_missing_per_sha1(sha1s_bin) + + # then + self.assertEquals(actual_content, [hashutil.hex_to_hash( + '745bab676c8f3cec8016e0c39ea61cf57e518865' + )]) + self.storage.content_missing_per_sha1.assert_called_with(sha1s_bin) + + @istest def origin_get(self): # given self.storage.origin_get = MagicMock(return_value={ diff --git a/swh/web/ui/tests/test_service.py b/swh/web/ui/tests/test_service.py --- a/swh/web/ui/tests/test_service.py +++ b/swh/web/ui/tests/test_service.py @@ -18,6 +18,54 @@ @patch('swh.web.ui.service.backend') @istest + def lookup_multiple_hashes_ball_missing(self, mock_backend): + # given + mock_backend.content_missing_per_sha1 = MagicMock(return_value=[]) + + # when + actual_lookup = service.lookup_multiple_hashes( + [{'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f'}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865'}]) + + # then + self.assertEquals(actual_lookup, [ + {'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', + 'found': True}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', + 'found': True} + ]) + + @patch('swh.web.ui.service.backend') + @istest + def lookup_multiple_hashes_some_missing(self, mock_backend): + # given + mock_backend.content_missing_per_sha1 = MagicMock(return_value=[ + hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f') + ]) + + # when + actual_lookup = service.lookup_multiple_hashes( + [{'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f'}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865'}]) + + # then + self.assertEquals(actual_lookup, [ + {'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', + 'found': False}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', + 'found': True} + ]) + + @patch('swh.web.ui.service.backend') + @istest def lookup_hash_does_not_exist(self, mock_backend): # given mock_backend.content_find = MagicMock(return_value=None) diff --git a/swh/web/ui/tests/views/test_browse.py b/swh/web/ui/tests/views/test_browse.py --- a/swh/web/ui/tests/views/test_browse.py +++ b/swh/web/ui/tests/views/test_browse.py @@ -25,11 +25,9 @@ # when rv = self.client.get('/search/') - self.assertEquals(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), '') + self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('messages'), []) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) + self.assertEqual(self.get_context_variable('responses'), []) self.assert_template_used('upload_and_search.html') @patch('swh.web.ui.views.browse.service') @@ -41,12 +39,13 @@ # when rv = self.client.get('/search/?q=sha1:456') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('q'), 'sha1:456') - self.assertEqual(self.get_context_variable('messages'), - ['Content with hash sha1:456 not found!']) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) + self.assertEqual(self.get_context_variable('messages'), []) + self.assertEqual(self.get_context_variable('responses'), [ + {'filename': 'User submitted hash', + 'sha1': 'sha1:456', + 'found': False}]) self.assert_template_used('upload_and_search.html') mock_service.lookup_hash.assert_called_once_with('sha1:456') @@ -60,12 +59,10 @@ # when rv = self.client.get('/search/?q=sha1_git:789') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('q'), 'sha1_git:789') - self.assertEqual(self.get_context_variable('messages'), - ['error msg']) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) + self.assertEqual(self.get_context_variable('messages'), ['error msg']) + self.assertEqual(self.get_context_variable('responses'), []) self.assert_template_used('upload_and_search.html') mock_service.lookup_hash.assert_called_once_with('sha1_git:789') @@ -79,90 +76,37 @@ # when rv = self.client.get('/search/?q=sha1:123') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('q'), 'sha1:123') - self.assertEqual(self.get_context_variable('messages'), - ['Content with hash sha1:123 found!']) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) + self.assertEqual(self.get_context_variable('messages'), []) + self.assertEqual(len(self.get_context_variable('responses')), 1) + resp = self.get_context_variable('responses')[0] + self.assertTrue(resp is not None) + self.assertEqual(resp['sha1'], 'sha1:123') + self.assertEqual(resp['found'], True) self.assert_template_used('upload_and_search.html') mock_service.lookup_hash.assert_called_once_with('sha1:123') @patch('swh.web.ui.views.browse.service') - @istest - def search_post_query_hash_not_found(self, mock_service): - # given - mock_service.lookup_hash.return_value = {'found': None} - - # when - rv = self.client.get('/search/?q=sha1:456') - - self.assertEquals(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), 'sha1:456') - self.assertEqual(self.get_context_variable('messages'), - ['Content with hash sha1:456 not found!']) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) - self.assert_template_used('upload_and_search.html') - - mock_service.lookup_hash.assert_called_once_with('sha1:456') - - @patch('swh.web.ui.views.browse.service') - @istest - def search_post_query_hash_bad_input(self, mock_service): - # given - mock_service.lookup_hash.side_effect = BadInputExc('error msg!') - - # when - rv = self.client.post('/search/', data=dict(q='sha1_git:987')) - - self.assertEquals(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), 'sha1_git:987') - self.assertEqual(self.get_context_variable('messages'), - ['error msg!']) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) - self.assert_template_used('upload_and_search.html') - - mock_service.lookup_hash.assert_called_once_with('sha1_git:987') - - @patch('swh.web.ui.views.browse.service') - @istest - def search_post_query_hash_found(self, mock_service): - # given - mock_service.lookup_hash.return_value = {'found': True} - - # when - rv = self.client.post('/search/', data=dict(q='sha1:321')) - - self.assertEquals(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), 'sha1:321') - self.assertEqual(self.get_context_variable('messages'), - ['Content with hash sha1:321 found!']) - self.assertEqual(self.get_context_variable('filename'), None) - self.assertEqual(self.get_context_variable('file'), None) - self.assert_template_used('upload_and_search.html') - - mock_service.lookup_hash.assert_called_once_with('sha1:321') - - @patch('swh.web.ui.views.browse.service') @patch('swh.web.ui.views.browse.request') @istest - def search_post_upload_and_hash_bad_input(self, mock_request, - mock_service): + def search_post_hashes_bad_input(self, mock_request, + mock_service): # given mock_request.data = {} mock_request.method = 'POST' - mock_request.files = dict(filename=FileMock('foobar')) - mock_service.upload_and_search.side_effect = BadInputExc( + mock_service.lookup_multiple_hashes.side_effect = BadInputExc( 'error bad input') # when (mock_request completes the post request) rv = self.client.post('/search/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) + self.assertEqual(self.get_context_variable('search_stats'), + {'nbfiles': 0, 'pct': 0}) + self.assertEqual(self.get_context_variable('responses'), []) self.assertEqual(self.get_context_variable('messages'), ['error bad input']) self.assert_template_used('upload_and_search.html') @@ -172,25 +116,34 @@ @patch('swh.web.ui.views.browse.service') @patch('swh.web.ui.views.browse.request') @istest - def search_post_upload_and_hash_not_found(self, mock_request, - mock_service): + def search_post_hashes_none(self, mock_request, mock_service): # given - mock_request.data = {} + mock_request.data = {'a': ['456caf10e9535160d90e874b45aa426de762f19f'], + 'b': ['745bab676c8f3cec8016e0c39ea61cf57e518865']} mock_request.method = 'POST' - mock_request.files = dict(filename=FileMock('foobar')) - mock_service.upload_and_search.return_value = {'filename': 'foobar', - 'sha1': 'blahhash', - 'found': False} + mock_service.lookup_multiple_hashes.return_value = [ + {'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', + 'found': False}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', + 'found': False} + ] # when (mock_request completes the post request) rv = self.client.post('/search/') # then - self.assertEquals(rv.status_code, 200) - self.assertEqual(self.get_context_variable('messages'), - ["File foobar with hash blahhash not found!"]) - self.assertEqual(self.get_context_variable('filename'), 'foobar') - self.assertEqual(self.get_context_variable('sha1'), 'blahhash') + self.assertEqual(rv.status_code, 200) + self.assertEqual(len(self.get_context_variable('responses')), 2) + self.assertTrue(self.get_context_variable('search_stats') is not None) + stats = self.get_context_variable('search_stats') + self.assertEqual(stats['nbfiles'], 2) + self.assertEqual(stats['pct'], 0) + a, b = self.get_context_variable('responses') + self.assertEqual(a['found'], False) + self.assertEqual(b['found'], False) + self.assertEqual(self.get_context_variable('messages'), []) self.assert_template_used('upload_and_search.html') mock_service.upload_and_search.called = True @@ -198,24 +151,34 @@ @patch('swh.web.ui.views.browse.service') @patch('swh.web.ui.views.browse.request') @istest - def search_post_upload_and_hash_found(self, mock_request, mock_service): + def search_post_hashes_some(self, mock_request, mock_service): # given - mock_request.data = {} + mock_request.data = {'a': '456caf10e9535160d90e874b45aa426de762f19f', + 'b': '745bab676c8f3cec8016e0c39ea61cf57e518865'} mock_request.method = 'POST' - mock_request.files = dict(filename=FileMock('foobar')) - mock_service.upload_and_search.return_value = {'filename': 'foobar', - 'sha1': '123456789', - 'found': True} + mock_service.lookup_multiple_hashes.return_value = [ + {'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', + 'found': False}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', + 'found': True} + ] # when (mock_request completes the post request) rv = self.client.post('/search/') # then - self.assertEquals(rv.status_code, 200) - self.assertEqual(self.get_context_variable('messages'), - ["File foobar with hash 123456789 found!"]) - self.assertEqual(self.get_context_variable('filename'), 'foobar') - self.assertEqual(self.get_context_variable('sha1'), '123456789') + self.assertEqual(rv.status_code, 200) + self.assertEqual(len(self.get_context_variable('responses')), 2) + self.assertTrue(self.get_context_variable('search_stats') is not None) + stats = self.get_context_variable('search_stats') + self.assertEqual(stats['nbfiles'], 2) + self.assertEqual(stats['pct'], 50) + self.assertEqual(self.get_context_variable('messages'), []) + a, b = self.get_context_variable('responses') + self.assertEqual(a['found'], False) + self.assertEqual(b['found'], True) self.assert_template_used('upload_and_search.html') mock_service.upload_and_search.called = True @@ -235,7 +198,7 @@ rv = self.client.get('/browse/content/sha1:sha1-hash/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('content.html') self.assertEqual(self.get_context_variable('message'), 'Not found!') @@ -255,7 +218,7 @@ rv = self.client.get('/browse/content/sha1:sha1-hash/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('content.html') self.assertEqual(self.get_context_variable('message'), 'Bad input!') @@ -280,7 +243,7 @@ rv = self.client.get('/browse/content/sha1:sha1-hash/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('content.html') self.assertIsNone(self.get_context_variable('message')) self.assertEqual(self.get_context_variable('content'), @@ -303,8 +266,8 @@ # when rv = self.client.get('/browse/content/sha1:sha1-hash/raw/') - self.assertEquals(rv.status_code, 200) - self.assertEquals(rv.data, stub_content_raw) + self.assertEqual(rv.status_code, 200) + self.assertEqual(rv.data, stub_content_raw) mock_urlfor.assert_called_once_with('api_content_raw', q='sha1:sha1-hash') @@ -326,7 +289,7 @@ rv = self.client.get('/browse/directory/sha2-invalid/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('directory.html') self.assertEqual(self.get_context_variable('message'), 'Invalid hash') @@ -344,7 +307,7 @@ rv = self.client.get('/browse/directory/some-sha1/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('directory.html') self.assertEqual(self.get_context_variable('message'), 'Listing for directory some-sha1:') @@ -383,7 +346,7 @@ rv = self.client.get('/browse/directory/some-sha1/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('directory.html') self.assertEqual(self.get_context_variable('message'), 'Listing for directory some-sha1:') @@ -410,7 +373,7 @@ rv = self.client.get('/browse/content/sha256:some-sha256/origin/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('content-with-origin.html') self.assertEqual(self.get_context_variable('message'), 'Not found!') @@ -429,7 +392,7 @@ rv = self.client.get('/browse/content/sha256:some-sha256/origin/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('content-with-origin.html') self.assertEqual( self.get_context_variable('message'), 'Invalid hash') @@ -453,7 +416,7 @@ rv = self.client.get('/browse/content/sha256:some-sha256/origin/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('content-with-origin.html') self.assertEqual( self.get_context_variable('message'), @@ -480,7 +443,7 @@ rv = self.client.get('/browse/origin/1/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('origin.html') self.assertEqual(self.get_context_variable('origin_id'), 1) self.assertEqual( @@ -499,7 +462,7 @@ rv = self.client.get('/browse/origin/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('origin.html') self.assertEqual(self.get_context_variable('origin_id'), 426) @@ -520,7 +483,7 @@ rv = self.client.get('/browse/origin/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('origin.html') self.assertEqual(self.get_context_variable('origin_id'), 426) self.assertEqual(self.get_context_variable('origin'), mock_origin) @@ -541,7 +504,7 @@ rv = self.client.get('/browse/person/1/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('person.html') self.assertEqual(self.get_context_variable('person_id'), 1) self.assertEqual( @@ -560,7 +523,7 @@ rv = self.client.get('/browse/person/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('person.html') self.assertEqual(self.get_context_variable('person_id'), 426) @@ -581,7 +544,7 @@ rv = self.client.get('/browse/person/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('person.html') self.assertEqual(self.get_context_variable('person_id'), 426) self.assertEqual(self.get_context_variable('person'), mock_person) @@ -602,7 +565,7 @@ rv = self.client.get('/browse/release/1/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('release.html') self.assertEqual(self.get_context_variable('sha1_git'), '1') self.assertEqual( @@ -621,7 +584,7 @@ rv = self.client.get('/browse/release/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('release.html') self.assertEqual(self.get_context_variable('sha1_git'), '426') @@ -669,7 +632,7 @@ rv = self.client.get('/browse/release/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('release.html') self.assertEqual(self.get_context_variable('sha1_git'), '426') self.assertEqual(self.get_context_variable('release'), @@ -691,7 +654,7 @@ rv = self.client.get('/browse/revision/1/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('sha1_git'), '1') self.assertEqual( @@ -711,7 +674,7 @@ rv = self.client.get('/browse/revision/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('sha1_git'), '426') self.assertEqual( @@ -774,7 +737,7 @@ rv = self.client.get('/browse/revision/426/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('sha1_git'), '426') self.assertEqual(self.get_context_variable('revision'), @@ -793,7 +756,7 @@ rv = self.client.get('/browse/revision/sha1/log/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-log.html') self.assertEqual(self.get_context_variable('sha1_git'), 'sha1') self.assertEqual( @@ -813,7 +776,7 @@ rv = self.client.get('/browse/revision/426/log/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-log.html') self.assertEqual(self.get_context_variable('sha1_git'), '426') self.assertEqual( @@ -854,7 +817,7 @@ rv = self.client.get('/browse/revision/426/log/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-log.html') self.assertEqual(self.get_context_variable('sha1_git'), '426') self.assertTrue( @@ -874,7 +837,7 @@ rv = self.client.get('/browse/revision/1/history/2/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('sha1_git_root'), '1') self.assertEqual(self.get_context_variable('sha1_git'), '2') @@ -896,7 +859,7 @@ rv = self.client.get('/browse/revision/321/history/654/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('sha1_git_root'), '321') self.assertEqual(self.get_context_variable('sha1_git'), '654') @@ -913,7 +876,7 @@ rv = self.client.get('/browse/revision/10/history/10/') # then - self.assertEquals(rv.status_code, 302) + self.assertEqual(rv.status_code, 302) @patch('swh.web.ui.views.browse.utils') @patch('swh.web.ui.views.browse.api') @@ -934,7 +897,7 @@ rv = self.client.get('/browse/revision/426/history/789/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('sha1_git_root'), '426') self.assertEqual(self.get_context_variable('sha1_git'), '789') @@ -955,7 +918,7 @@ rv = self.client.get('/browse/revision/1/directory/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('sha1_git'), '1') self.assertEqual(self.get_context_variable('path'), '.') @@ -977,7 +940,7 @@ rv = self.client.get('/browse/revision/10/directory/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('sha1_git'), '10') self.assertEqual(self.get_context_variable('path'), '.') @@ -1034,7 +997,7 @@ rv = self.client.get('/browse/revision/100/directory/some/path/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('sha1_git'), '100') self.assertEqual(self.get_context_variable('revision'), '100') @@ -1056,7 +1019,7 @@ rv = self.client.get('/browse/revision/123/history/456/directory/a/b/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('sha1_git_root'), '123') self.assertEqual(self.get_context_variable('sha1_git'), '456') @@ -1078,7 +1041,7 @@ rv = self.client.get('/browse/revision/123/history/456/directory/a/c/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('sha1_git_root'), '123') self.assertEqual(self.get_context_variable('sha1_git'), '456') @@ -1097,7 +1060,7 @@ rv = self.client.get('/browse/revision/1/history/2/directory/path/to') # then - self.assertEquals(rv.status_code, 301) + self.assertEqual(rv.status_code, 301) @patch('swh.web.ui.views.browse.service') @istest @@ -1107,7 +1070,7 @@ rv = self.client.get('/browse/revision/1/history/1/directory/path/to') # then - self.assertEquals(rv.status_code, 301) + self.assertEqual(rv.status_code, 301) @patch('swh.web.ui.views.browse.api') @istest @@ -1140,7 +1103,7 @@ 'path/to/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('sha1_git_root'), '100') self.assertEqual(self.get_context_variable('sha1_git'), '999') @@ -1164,7 +1127,7 @@ '/history/123/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertIsNone(self.get_context_variable('revision')) self.assertEqual(self.get_context_variable('message'), @@ -1185,7 +1148,7 @@ 'branch/dev/history/123/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertIsNone(self.get_context_variable('revision')) self.assertEqual(self.get_context_variable('message'), @@ -1208,7 +1171,7 @@ '/history/789/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertIsNone(self.get_context_variable('revision')) self.assertEqual(self.get_context_variable('message'), @@ -1232,7 +1195,7 @@ rv = self.client.get('/browse/revision/origin/99/history/123/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('revision'), stub_rev) self.assertIsNone(self.get_context_variable('message')) @@ -1250,7 +1213,7 @@ # when rv = self.client.get('/browse/revision/origin/1/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertIsNone(self.get_context_variable('revision')) self.assertEqual(self.get_context_variable('message'), 'Not found') @@ -1268,7 +1231,7 @@ # when rv = self.client.get('/browse/revision/origin/1000/branch/dev/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertIsNone(self.get_context_variable('revision')) self.assertEqual(self.get_context_variable('message'), 'Bad Input') @@ -1288,7 +1251,7 @@ '/branch/scratch/master' '/ts/1990-01-10/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertIsNone(self.get_context_variable('revision')) self.assertEqual(self.get_context_variable('message'), 'Other') @@ -1308,7 +1271,7 @@ # when rv = self.client.get('/browse/revision/origin/1/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision.html') self.assertEqual(self.get_context_variable('revision'), stub_rev) self.assertIsNone(self.get_context_variable('message')) @@ -1327,7 +1290,7 @@ rv = self.client.get('/browse/revision/origin/2' '/directory/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertIsNone(self.get_context_variable('result')) self.assertEqual(self.get_context_variable('message'), @@ -1347,7 +1310,7 @@ rv = self.client.get('/browse/revision/origin/2' '/directory/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertIsNone(self.get_context_variable('result')) self.assertEqual(self.get_context_variable('message'), 'Bad Robot') @@ -1366,7 +1329,7 @@ rv = self.client.get('/browse/revision/origin/2' '/directory/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertIsNone(self.get_context_variable('result')) self.assertEqual(self.get_context_variable('message'), @@ -1391,7 +1354,7 @@ '/ts/2013-20-20 10:02' '/directory/some/file/') - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertEqual(self.get_context_variable('result'), stub_res) self.assertIsNone(self.get_context_variable('message')) @@ -1412,7 +1375,7 @@ '/directory/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertIsNone(self.get_context_variable('result')) self.assertEqual(self.get_context_variable('message'), 'Not found!') @@ -1436,7 +1399,7 @@ '/directory/some/path/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') self.assertIsNone(self.get_context_variable('result')) self.assertEqual(self.get_context_variable('message'), @@ -1463,9 +1426,9 @@ '/directory/emacs-24.5/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('revision-directory.html') - self.assertEquals(self.get_context_variable('result'), stub_dir) + self.assertEqual(self.get_context_variable('result'), stub_dir) self.assertIsNone(self.get_context_variable('message')) self.assertEqual(self.get_context_variable('path'), 'emacs-24.5') @@ -1487,7 +1450,7 @@ rv = self.client.get('/browse/entity/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('entity.html') self.assertEqual(self.get_context_variable('entities'), []) self.assertEqual(self.get_context_variable('message'), 'Not found!') @@ -1505,7 +1468,7 @@ rv = self.client.get('/browse/entity/blah-blah-uuid/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('entity.html') self.assertEqual(self.get_context_variable('entities'), []) self.assertEqual(self.get_context_variable('message'), 'wrong input!') @@ -1526,7 +1489,7 @@ '5f4d4c51-5a9b-4e28-88b3-b3e4e8396cba/') # then - self.assertEquals(rv.status_code, 200) + self.assertEqual(rv.status_code, 200) self.assert_template_used('entity.html') self.assertEqual(self.get_context_variable('entities'), stub_entities) self.assertIsNone(self.get_context_variable('message')) diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py --- a/swh/web/ui/views/api.py +++ b/swh/web/ui/views/api.py @@ -5,6 +5,7 @@ from types import GeneratorType + from flask import request, url_for, Response, redirect from swh.web.ui import service, utils diff --git a/swh/web/ui/views/browse.py b/swh/web/ui/views/browse.py --- a/swh/web/ui/views/browse.py +++ b/swh/web/ui/views/browse.py @@ -25,69 +25,68 @@ One form to submit either: - hash query to look up in swh storage - - some file content to upload, compute its hash and look it up in swh - storage + - file hashes calculated client-side to be queried in swh storage - both Returns: dict representing data to look for in swh storage. The following keys are returned: - - file: File submitted for upload - - filename: Filename submitted for upload - - q: Query on hash to look for - - message: Message detailing if data has been found or not. - + - search_stats: {'nbfiles': X, 'pct': Y} the number of total + queried files and percentage of files not in storage respectively + - responses: array of {'filename': X, 'sha1': Y, 'found': Z} + - messages: General messages. + TODO: + Batch-process with all checksums, not just sha1 """ - env = {'filename': None, - 'q': None, - 'file': None} - data = None - q = env['q'] - file = env['file'] - - if request.method == 'GET': - data = request.args - elif request.method == 'POST': - data = request.data - # or hash and search a file - file = request.files.get('filename') - - # could either be a query for sha1 hash - q = data.get('q') + env = {'q': None, + 'search_stats': None, + 'responses': None, + 'messages': []} + search_stats = None + responses = [] messages = [] - if q: + # Get with a single hash request + if request.method == 'GET': + data = request.args + q = data.get('q') env['q'] = q - - try: - r = service.lookup_hash(q) - messages.append('Content with hash %s%sfound!' % ( - q, ' ' if r.get('found') else ' not ')) - except BadInputExc as e: - messages.append(str(e)) - - if file and file.filename: - env['file'] = file - try: - uploaded_content = service.upload_and_search(file) - filename = uploaded_content['filename'] - sha1 = uploaded_content['sha1'] - found = uploaded_content['found'] - - messages.append('File %s with hash %s%sfound!' % ( - filename, sha1, ' ' if found else ' not ')) - - env.update({ - 'filename': filename, - 'sha1': sha1, - }) - except BadInputExc as e: - messages.append(str(e)) - - env['q'] = q if q else '' + if q: + try: + search_stats = {'nbfiles': 0, 'pct': 0} + r = service.lookup_hash(q) + responses.append({'filename': 'User submitted hash', + 'sha1': q, + 'found': r.get('found') is not None}) + search_stats['nbfiles'] = 1 + search_stats['pct'] = 100 if r.get('found') is not None else 0 + except BadInputExc as e: + messages.append(str(e)) + + # POST form submission with many hash requests + elif request.method == 'POST': + data = request.form + search_stats = {'nbfiles': 0, 'pct': 0} + queries = [] + # Remove potential inputs with no associated value + for k, v in data.items(): + if v is not None and v != '': + queries.append({'filename': k, 'sha1': v}) + + if len(queries) > 0: + try: + lookup = service.lookup_multiple_hashes(queries) + nbfound = len([x for x in lookup if x['found']]) + responses = lookup + search_stats['nbfiles'] = len(queries) + search_stats['pct'] = (nbfound / len(queries))*100 + except BadInputExc as e: + messages.append(str(e)) + + env['search_stats'] = search_stats + env['responses'] = responses env['messages'] = messages - return render_template('upload_and_search.html', **env)